<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>36.6. Character Set</title>
<link rel="stylesheet" href="dbstyle.css" type="text/css">
<meta name="generator" content="DocBook XSL Stylesheets V1.72.0">
<link rel="start" href="index.html" title="Programmer's Reference Guide">
<link rel="up" href="zend.search.lucene.html" title="Chapter 36. Zend_Search_Lucene">
<link rel="prev" href="zend.search.lucene.query-api.html" title="36.5. Query Construction API">
<link rel="next" href="zend.search.lucene.extending.html" title="36.7. Extensibility">
<link rel="chapter" href="introduction.html" title="Chapter 1. Introduction to Zend Framework">
<link rel="chapter" href="zend.acl.html" title="Chapter 2. Zend_Acl">
<link rel="chapter" href="zend.auth.html" title="Chapter 3. Zend_Auth">
<link rel="chapter" href="zend.cache.html" title="Chapter 4. Zend_Cache">
<link rel="chapter" href="zend.config.html" title="Chapter 5. Zend_Config">
<link rel="chapter" href="zend.console.getopt.html" title="Chapter 6. Zend_Console_Getopt">
<link rel="chapter" href="zend.controller.html" title="Chapter 7. Zend_Controller">
<link rel="chapter" href="zend.currency.html" title="Chapter 8. Zend_Currency">
<link rel="chapter" href="zend.date.html" title="Chapter 9. Zend_Date">
<link rel="chapter" href="zend.db.html" title="Chapter 10. Zend_Db">
<link rel="chapter" href="zend.debug.html" title="Chapter 11. Zend_Debug">
<link rel="chapter" href="zend.dojo.html" title="Chapter 12. Zend_Dojo">
<link rel="chapter" href="zend.dom.html" title="Chapter 13. Zend_Dom">
<link rel="chapter" href="zend.exception.html" title="Chapter 14. Zend_Exception">
<link rel="chapter" href="zend.feed.html" title="Chapter 15. Zend_Feed">
<link rel="chapter" href="zend.filter.html" title="Chapter 16. Zend_Filter">
<link rel="chapter" href="zend.form.html" title="Chapter 17. Zend_Form">
<link rel="chapter" href="zend.gdata.html" title="Chapter 18. Zend_Gdata">
<link rel="chapter" href="zend.http.html" title="Chapter 19. Zend_Http">
<link rel="chapter" href="zend.infocard.html" title="Chapter 20. Zend_InfoCard">
<link rel="chapter" href="zend.json.html" title="Chapter 21. Zend_Json">
<link rel="chapter" href="zend.layout.html" title="Chapter 22. Zend_Layout">
<link rel="chapter" href="zend.ldap.html" title="Chapter 23. Zend_Ldap">
<link rel="chapter" href="zend.loader.html" title="Chapter 24. Zend_Loader">
<link rel="chapter" href="zend.locale.html" title="Chapter 25. Zend_Locale">
<link rel="chapter" href="zend.log.html" title="Chapter 26. Zend_Log">
<link rel="chapter" href="zend.mail.html" title="Chapter 27. Zend_Mail">
<link rel="chapter" href="zend.measure.html" title="Chapter 28. Zend_Measure">
<link rel="chapter" href="zend.memory.html" title="Chapter 29. Zend_Memory">
<link rel="chapter" href="zend.mime.html" title="Chapter 30. Zend_Mime">
<link rel="chapter" href="zend.openid.html" title="Chapter 31. Zend_OpenId">
<link rel="chapter" href="zend.paginator.html" title="Chapter 32. Zend_Paginator">
<link rel="chapter" href="zend.pdf.html" title="Chapter 33. Zend_Pdf">
<link rel="chapter" href="zend.registry.html" title="Chapter 34. Zend_Registry">
<link rel="chapter" href="zend.rest.html" title="Chapter 35. Zend_Rest">
<link rel="chapter" href="zend.search.lucene.html" title="Chapter 36. Zend_Search_Lucene">
<link rel="chapter" href="zend.server.html" title="Chapter 37. Zend_Server">
<link rel="chapter" href="zend.service.html" title="Chapter 38. Zend_Service">
<link rel="chapter" href="zend.session.html" title="Chapter 39. Zend_Session">
<link rel="chapter" href="zend.soap.html" title="Chapter 40. Zend_Soap">
<link rel="chapter" href="zend.test.html" title="Chapter 41. Zend_Test">
<link rel="chapter" href="zend.text.html" title="Chapter 42. Zend_Text">
<link rel="chapter" href="zend.timesync.html" title="Chapter 43. Zend_TimeSync">
<link rel="chapter" href="zend.translate.html" title="Chapter 44. Zend_Translate">
<link rel="chapter" href="zend.uri.html" title="Chapter 45. Zend_Uri">
<link rel="chapter" href="zend.validate.html" title="Chapter 46. Zend_Validate">
<link rel="chapter" href="zend.version.html" title="Chapter 47. Zend_Version">
<link rel="chapter" href="zend.view.html" title="Chapter 48. Zend_View">
<link rel="chapter" href="zend.xmlrpc.html" title="Chapter 49. Zend_XmlRpc">
<link rel="appendix" href="requirements.html" title="Appendix A. Zend Framework Requirements">
<link rel="appendix" href="coding-standard.html" title="Appendix B. Zend Framework Coding Standard for PHP">
<link rel="appendix" href="copyrights.html" title="Appendix C. Copyright Information">
<link rel="index" href="the.index.html" title="Index">
<link rel="subsection" href="zend.search.lucene.charset.html#zend.search.lucene.charset.description" title="36.6.1. UTF-8 and single-byte character set support">
<link rel="subsection" href="zend.search.lucene.charset.html#zend.search.lucene.charset.default_analyzer" title="36.6.2. Default text analyzer.">
<link rel="subsection" href="zend.search.lucene.charset.html#zend.search.lucene.charset.utf_analyzer" title="36.6.3. UTF-8 compatible text analyzers.">
</head>
<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
<div class="navheader"><table width="100%" summary="Navigation header">
<tr><th colspan="3" align="center">36.6. Character Set</th></tr>
<tr>
<td width="20%" align="left">
<a accesskey="p" href="zend.search.lucene.query-api.html">Prev</a> </td>
<th width="60%" align="center">Chapter 36. Zend_Search_Lucene</th>
<td width="20%" align="right"> <a accesskey="n" href="zend.search.lucene.extending.html">Next</a>
</td>
</tr>
</table></div>
<div class="sect1" lang="en">
<div class="titlepage"><div><div><h2 class="title" style="clear: both">
<a name="zend.search.lucene.charset"></a>36.6. Character Set</h2></div></div></div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.charset.description"></a>36.6.1. UTF-8 and single-byte character set support</h3></div></div></div>
<p>
            Zend_Search_Lucene works with the UTF-8 charset internally. Index files store
            unicode data in Java's "modified UTF-8 encoding". Zend_Search_Lucene core
            completely supports this encoding with one exception.
            <sup>[<a name="id3083746" href="#ftn.id3083746">8</a>]</sup>
        </p>
<p>
            Actual input data encoding may be specified through Zend_Search_Lucene API. Data will
            be automatically converted into UTF-8 encoding.
        </p>
</div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.charset.default_analyzer"></a>36.6.2. Default text analyzer.</h3></div></div></div>
<p>
            However, the default text analyzer (which is also used within query parser) uses
            ctype_alpha() for tokenizing text and queries.
        </p>
<p>
            ctype_alpha() is not UTF-8 compatible, so the analyzer converts text to 'ASCII//TRANSLIT' encoding before
            indexing. The same processing is transparently performed during query parsing.
            <sup>[<a name="id3078867" href="#ftn.id3078867">9</a>]</sup>
        </p>
<div class="note"><table border="0" summary="Note: ">
<tr>
<td rowspan="2" align="center" valign="top" width="25"><img alt="[Note]" src="images/note.png"></td>
<th align="left"></th>
</tr>
<tr><td align="left" valign="top"><p>
                Default analyzer doesn't treats numbers as parts of terms. Use corresponding 'Num' analyzer if you don't want words 
                to be broken by numbers.  
            </p></td></tr>
</table></div>
</div>
<div class="sect2" lang="en">
<div class="titlepage"><div><div><h3 class="title">
<a name="zend.search.lucene.charset.utf_analyzer"></a>36.6.3. UTF-8 compatible text analyzers.</h3></div></div></div>
<p>
            Zend_Search_Lucene also contains a set of UTF-8 compatible analyzers: <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8</code>,
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num</code>, <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive</code>,
            <code class="code">Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive</code>.
        </p>
<p>
             Any of this analyzers can be enabled with the code like this:

            </p>
<pre class="programlisting">&lt;?php
Zend_Search_Lucene_Analysis_Analyzer::setDefault(
    new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8());
            </pre>
<p>
        </p>
<div class="warning"><table border="0" summary="Warning: ">
<tr>
<td rowspan="2" align="center" valign="top" width="25"><img alt="[Warning]" src="images/warning.png"></td>
<th align="left"></th>
</tr>
<tr><td align="left" valign="top">
<p>
                UTF-8 compatible analyzers were improved in ZF 1.5. Early versions of analyzers assumed 
                all non-ascii characters are letters. New analyzers implementation has more accurate behavior. 
            </p>
<p>
                This may need you to re-build index to have data and search queries tokenized in the same way, otherwise search engine
                may return wrong result sets.  
            </p>
</td></tr>
</table></div>
<p>
            All of these analyzers need PCRE (Perl-compatible regular expressions) library to be compiled with UTF-8 support turned on. 
            PCRE UTF-8 support is turned on for the PCRE library sources bandled with PHP source code distribution, but if shared libraru is used
            instead of bandled with PHP sources, then UTF-8 support state may depend on you operation system. 
        </p>
<p>
             Use the following code to check, if PCRE UTF-8 suppor is enabled:

            </p>
<pre class="programlisting">&lt;?php
if (@preg_match('/\pL/u', 'a') == 1) {
    echo "PCRE unicode support is turned on.\n";
} else {
    echo "PCRE unicode support is turned off.\n";
}
            </pre>
<p>
        </p>
<p>
            Case insensitive versions of UTF-8 compatible analyzers also need <a href="http://www.php.net/manual/en/ref.mbstring.php" target="_top">mbstring</a> extension to be enabled. 
        </p>
<p>
            If you don't want mbstring extension to be turned on, but need case insensitive search, you may use the following approach: normalize source data before indexing
            and query string before searching by converting them to lowercase:
        
            </p>
<pre class="programlisting">&lt;?php
// Indexing
setlocale(LC_CTYPE, 'de_DE.iso-8859-1');

...

Zend_Search_Lucene_Analysis_Analyzer::setDefault(
    new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8());

...

$doc = new Zend_Search_Lucene_Document();

$doc-&gt;addField(Zend_Search_Lucene_Field::UnStored('contents', strtolower($contents)));

// Title field for search through (indexed, unstored)
$doc-&gt;addField(Zend_Search_Lucene_Field::UnStored('title', strtolower($title)));

// Title field for retrieving (unindexed, stored)
$doc-&gt;addField(Zend_Search_Lucene_Field::UnIndexed('_title', $title));
            </pre>
<p>

            </p>
<pre class="programlisting">&lt;?php
// Searching
setlocale(LC_CTYPE, 'de_DE.iso-8859-1');

...

Zend_Search_Lucene_Analysis_Analyzer::setDefault(
    new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8());

...

$hits = $index-&gt;find(strtolower($query));
            </pre>
<p>
        </p>
</div>
<div class="footnotes">
<br><hr width="100" align="left">
<div class="footnote">
<p><sup>[<a name="ftn.id3083746" href="#id3083746">8</a>] </sup>
                   Zend_Search_Lucene supports only Basic Multilingual Plane
                   (BMP) characters (from 0x0000 to 0xFFFF) and doesn't support
                   "supplementary characters" (characters whose code points are
                   greater than 0xFFFF)
               </p>
<p>
                   Java 2 represents these characters as a pair of char (16-bit)
                   values, the first from the high-surrogates range (0xD800-0xDBFF),
                   the second from the low-surrogates range (0xDC00-0xDFFF). Then
                   they are encoded as usual UTF-8 characters in six bytes.
                   Standard UTF-8 representation uses four bytes for supplementary
                   characters.
               </p>
</div>
<div class="footnote"><p><sup>[<a name="ftn.id3078867" href="#id3078867">9</a>] </sup>
                   Conversion to 'ASCII//TRANSLIT' may depend on current locale and OS.
               </p></div>
</div>
</div>
<div class="navfooter"><table width="100%" summary="Navigation footer">
<tr>
<td width="40%" align="left">
<a accesskey="p" href="zend.search.lucene.query-api.html">Prev</a> </td>
<td width="20%" align="center"><a accesskey="u" href="zend.search.lucene.html">Up</a></td>
<td width="40%" align="right"> <a accesskey="n" href="zend.search.lucene.extending.html">Next</a>
</td>
</tr>
<tr>
<td width="40%" align="left" valign="top">36.5. Query Construction API </td>
<td width="20%" align="center"><a accesskey="h" href="index.html">Home</a></td>
<td width="40%" align="right" valign="top"> 36.7. Extensibility</td>
</tr>
</table></div>
<div class="revinfo"></div>
</body>
</html>
